package club.zynight.crawler.main;

import club.zynight.crawler.entity.Location;
import club.zynight.crawler.entity.SenicSpotEntity;
import club.zynight.crawler.utils.DataUtils;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.http.HttpHost;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

public class Es {
    public static final String HOST = "192.168.201.164";
    public static final int PORT = 9200;
    public static final String INDEX = "geography";
    public static void main(String[] args) throws Exception {
        toEs();
    }
    public static void toEs(){
        RestHighLevelClient client = null;
        try{
            //验证是否联通和索引是否存在
            if((client=isConnected())==null) {
                throw  new Exception("连接失败");
            }
            for(String province :getProvince()){//获取所有的省份
                String url = "http://www.poilist.cn/poi-list-景点-"+province+"/";
                BulkRequest request = getListBulk(getProvinceData(url)); //获取当前省份的所有数据
                BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);
                System.out.println("插入"+province+"所有景点数据耗时:"+response.getTook());
            }
        }catch (Exception e){e.printStackTrace();}
        finally {//最后总要自己关闭client对象
            try {
                client.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }


    public static List<String> getProvince() throws Exception {
        Document document = Jsoup.parse(new URL("http://www.poilist.cn/cities-list/景点"),1000*60);
        Elements elements = document.getElementsByTag("li");
        return manageProvince(elements);
    }


    public static RestHighLevelClient isConnected(){
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost(HOST,PORT,"http"))
        );
        try {
            client.indices().get(new GetIndexRequest(INDEX),RequestOptions.DEFAULT);
        } catch (Exception e) {
            try {
                System.out.println("创建索引");
                client.indices().create(new CreateIndexRequest(INDEX),RequestOptions.DEFAULT);
            } catch (IOException ex) {
               return null;
            }
        }
        return client;
    }

    public static List<SenicSpotEntity> getProvinceData(String url) throws Exception {
        List<SenicSpotEntity> list = new ArrayList();
        for(int start = 0 ; ;start+=30){
            Document document = Jsoup.parse(new URL(url+start+"/"),1000*60);
            Elements elements = document.getElementsByTag("tr");
            if(elements.size()==1) break;//当前景点数据已全部获取
            list.addAll(manageProvinceData(elements));
        }
        System.out.println("===================="+url+":"+list.size()+"========================");
        return list;
    }


    public static List<SenicSpotEntity>  manageProvinceData(Elements elements) throws Exception {
        elements.remove(0);//删除第一行的标题栏
        List<SenicSpotEntity> list = new ArrayList();
        for(Element element : elements){//
            Elements infos = element.getElementsByTag("td");
            SenicSpotEntity entity = new SenicSpotEntity();
            Location location = new Location();
            int index = 0;
            for(Element info : infos){//获取当前行中的每一列的数据
                DataUtils.setEsSource(index++,info.text(),entity,location);//存储数据
            }
            entity.setLocation(location);
            list.add(entity);
        }
        return list;
    }

    public static List<String> manageProvince(Elements elements){
        List<String> list = new ArrayList();
        for (Element element :elements){
            String province = element.getElementsByTag("a").first().text();
            list.add(province);
        }
        return list;
    }


    public static BulkRequest getListBulk(List<SenicSpotEntity> list) throws JsonProcessingException {
        BulkRequest request = new BulkRequest();
        Iterator<SenicSpotEntity> it = list.iterator();
        while(it.hasNext()){
            SenicSpotEntity entity = it.next();
            request.add(new IndexRequest().index(INDEX).source(new ObjectMapper().writeValueAsString(entity),XContentType.JSON));;
        }
        return request;
    }

    public static void testOne(){
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost("192.168.201.164",9200,"http"))
        );
        try {
            SenicSpotEntity entity = new SenicSpotEntity();
            Location location = new Location();
            entity.setName("和平广场");
            entity.setCity("黑龙江");
            entity.setCounty("齐齐哈尔");
            location.setLon(123.917221);
            location.setLat(47.360886);
            entity.setLocation(location);
            String value = new ObjectMapper().writeValueAsString(entity);
            System.out.println(value);
            BulkRequest request = new BulkRequest();
            request.add(new IndexRequest().index("geography").source(value,XContentType.JSON));
            BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);
            System.out.println("耗时:"+response.getTook());

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                client.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}
